package com.trainoo.job;

import com.trainoo.crawler.NovelCrawler;
import com.trainoo.service.ChapterService;
import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

/**
 * Created by zhoutao on 2018/6/28 16:46
 */
@Component
public class NovelCrawlerJob {

    @Autowired
    private ChapterService chapterService;

    private static Logger LOG = LoggerFactory.getLogger(NovelCrawlerJob.class);

    private static int numberOfCrawlers = 1;
    private static int maxDepthOfCrawling = 1;
    private static String crawlStorageFolder = "E:/crawl";
    private static String URL = "http://www.6mao.com/html/53/53302/index.html";

    public void start() throws Exception {
        CrawlConfig config = new CrawlConfig();           // 定义爬虫配置
        config.setCrawlStorageFolder(crawlStorageFolder); // 设置爬虫文件存储位置
        config.setMaxDepthOfCrawling(maxDepthOfCrawling); // 设置爬虫爬取深度
        config.setResumableCrawling(true);                // 恢复停止/崩溃的爬虫

        PageFetcher pageFetcher = new PageFetcher(config);       // 实例化页面获取器
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig(); // 实例化爬虫机器人配置 比如可以设置 user-agent

        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

        controller.addSeed(URL);

        controller.start(NovelCrawler.class, numberOfCrawlers);
    }

    @Scheduled(cron = "0 * * * * ?")
    public void run(){
        try {
            LOG.info("====开始=====");
            start();
            LOG.info("====结束=====");
        } catch (Exception e) {
            LOG.info("出现异常，", e);
        }
    }
}
